from scrapy.spiders import Spider
from scrapy import Request
import json
from ..items import create_item_class
import logging


class StockListSpider(Spider):
    name = 'list'
    hq_url = 'http://money.finance.sina.com.cn/d/api/openapi_proxy.php' \
             '/?__s=[["hq","hs_a","",0,{},40]]'
    hk_url = 'http://money.finance.sina.com.cn/d/api/openapi_proxy.php' \
             '/?__s=[["hk","qbgg_hk","",0,{},40]]'
    us_url = 'http://money.finance.sina.com.cn/d/api/openapi_proxy.php/' \
             '?__s=[["us_c",0,"","",0,{},40]]'
    hq_page_number = 0
    hk_page_number = 0
    us_page_number = 0
    custom_settings = {
        'ITEM_PIPELINES': {
            'newscrawler.pipelines.StockPipeline': 300
        }
    }

    def start_requests(self):
        yield Request(self.hq_url.format(self.hq_page_number), callback=self.parse_list, meta={'market': 'hs'})
        yield Request(self.hk_url.format(self.hk_page_number), callback=self.parse_list, meta={'market': 'hk'})
        yield Request(self.us_url.format(self.us_page_number), callback=self.parse_list, meta={'market': 'us'})

    def parse_list(self, response):
        market = response.meta.get('market')
        content = response.body
        content = content[1:-1]
        data_dict = json.loads(content)
        code = data_dict['code']
        count = int(data_dict['count'])
        fields = data_dict['fields']
        items = data_dict['items']
        MyItem = create_item_class('StockItem', fields + ['market'])
        if len(items) <= 0:
            return
        for i in items:
            item = MyItem()
            for k, v in zip(fields, i):
                item[k] = v
            item['market'] = market
            yield item
        if market == 'hs' and self.hq_page_number * 40 < count:
            self.hq_page_number += 1
            yield Request(self.hq_url.format(self.hq_page_number), callback=self.parse_list, meta={'market': 'hs'})
        elif market == 'hk' and self.hk_page_number * 40 < count:
            self.hk_page_number += 1
            yield Request(self.hk_url.format(self.hk_page_number), callback=self.parse_list, meta={'market': 'hk'})
        elif market == 'us' and self.us_page_number * 40 < count:
            self.us_page_number += 1
            yield Request(self.us_url.format(self.us_page_number), callback=self.parse_list, meta={'market': 'us'})
